import nltk
import pandas as pd
from gensim.utils import tokenize
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

nltk.download('stopwords')

!pip install vaderSentiment
from nltk.corpus import stopwords
# from nltk.sentiment import SentimentIntensityAnalyzer


# Reading the Dataset
df = pd.read_csv("Data/skytrax_reviews.csv")
df

# Checking Dataset distribution
df[df['Recommended'] == 'no'].count()

df[df['Recommended'] == 'yes'].count()

[out] Positive Reviews = 25.19%

Negative Reviews = 74.81%

#Extracting Only Useful Columns
review_df = df[['Review Text', 'Recommended']]
review_df

# Preprocessing reviews Text
def _process_review(review):
    review = review.lower()
    review = list(tokenize(review))
    
    stop_words = set(stopwords.words('english')) 
    
    review = ' '.join([word for word in review if word not in stop_words])
    review = ''.join([i for i in review if not i.isdigit()]).strip()
    return review
	
	def process_reviews(list_of_reviews):
    processed_reviews = []
    for review in list_of_reviews:
        processed_reviews.append(_process_review(review))
    return processed_reviews
	
review_df['processed_review_text'] = process_reviews(review_df['Review Text'])

# Applying Sentiment Score

vader_analyzer = SentimentIntensityAnalyzer()
def sentiment_analyzer_scores(sentence):
    """ Applies rule based sentiment analysis to review text """
    score = vader_analyzer.polarity_scores(sentence)
    return score['compound']
	review_df['sentiment_score'] = review_df['processed_review_text'].apply(sentiment_analyzer_scores)
	review_df.to_csv('Data\sentiment_scores.csv', index=False)
	
# For sentiemnt classification
	
	def score_limit(review_score):
    if review_score >= 0.0:
        return "yes"
    else:
        return "no"
		review_df['predicted'] = review_df['sentiment_score'].apply(score_limit)
		review_df
		
#For analysis evaluation
 	
		from sklearn.metrics import confusion_matrix

confusion_matrix(review_df['Recommended'], review_df['predicted'])
array([[13918, 15146],
       [  912,  8878]], dtype=int64)
	   
	   from sklearn.metrics import classification_report

print(classification_report(review_df['Recommended'], review_df['predicted']))
 precision    recall  f1-score   support

          no       0.94      0.48      0.63     29064
         yes       0.37      0.91      0.53      9790

    accuracy                           0.59     38854
   macro avg       0.65      0.69      0.58     38854
weighted avg       0.80      0.59      0.61     38854

# For VADER score distribution figures
# with bin size "10"

plt.rcParams.update({'font.size': 18})
plt.figure(figsize=[14, 10])
plt.hist(review_df['sentiment_score'], bins=10)
plt.ylabel("Frequency of Reviews in 10 bins")
plt.savefig("hist10bins.png")
plt.show()

# with bin size "20"
plt.figure(figsize=[14, 10])
plt.hist(review_df['sentiment_score'], bins=20)
plt.ylabel("Frequency of Reviews in 20 bins")
plt.savefig("hist20bins.png")
plt.show()

# with bin size "30"
plt.figure(figsize=[14, 10])
plt.hist(review_df['sentiment_score'], bins=30)
plt.ylabel("Frequency of Reviews in 30 bins")
plt.savefig("hist30bins.png")
plt.show()